#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@File    :   country_asn.py    
@Contact :   291622538@qq.com

@Modify Time      @Author    @Version    @Description
------------      -------    --------    -----------
2020/10/25 17:00   fan        1.0         None
'''
from scrapy import Spider, Request
from website.items import *


class CountryASN(Spider):
    # 爬虫名称
    name = "country_asn_spider"
    # 专用的通道
    custom_settings = {
        'ITEM_PIPELINES': {'website.pipelines.SaveCountryASNPipeline': 301},
    }

    # 爬虫入口
    def start_requests(self):

        URL = "https://whois.ipip.net/countries"
        # dont_filter = False 相同的url只请求一次
        yield Request(URL, callback=self.country_asn_parse, dont_filter=False)

    def country_asn_parse(self, response):
        country_asn_list = response.xpath('//table/tr')
        # print(len(country_asn_list))
        for num, country_asn in enumerate(country_asn_list, 1):
            # if num > 1:
            #     break

            td = country_asn.xpath('td')
            countryName = "".join(td[0].xpath('a/text()').extract())
            countryHref = "".join(td[0].xpath('a/@href').extract())
            countryCode = countryHref.split("/")[-1]
            allocatedASNs = "".join(td[1].xpath('./text()').extract())
            announcedASNs = "".join(td[2].xpath('./text()').extract())
            asnIpv4Number = "".join(td[3].xpath('./text()').extract())
            asnIpv6Number = "".join(td[4].xpath('./text()').extract())
            countryHref = response.urljoin(countryHref)
            countryASNItem = CountryASNItem(countryName=countryName, countryHref=countryHref, countryCode=countryCode,
                                            allocatedASNs=allocatedASNs,
                                            announcedASNs=announcedASNs, asnIpv4Number=asnIpv4Number,
                                            asnIpv6Number=asnIpv6Number)

            yield countryASNItem
            # yield Request(countryHref, dont_filter=False,
            #               callback=lambda response, countryCode=countryCode: self.asn_list_parse(response, countryCode))
            yield Request(countryHref, meta={"countryCode": countryCode}, callback=self.asn_list_parse,
                          dont_filter=False)

    def asn_list_parse(self, response):
        asn_list = response.xpath('//table/tr')
        # print(len(asn_list))
        for num, country_asn in enumerate(asn_list, 1):
            # if num > 10:
            #     break

            td = country_asn.xpath('td')
            asnNum = "".join(td[0].xpath('a/text()').extract())
            asnNumHref = "".join(td[0].xpath('a/@href').extract())
            asnName = "".join(td[1].xpath('./text()').extract())
            ipv4NumIps = "".join(td[2].xpath('./text()').extract())
            ipv6NumIps = "".join(td[3].xpath('./text()').extract())
            asnNumHref = response.urljoin(asnNumHref)
            asnItem = ASNItem(asnNum=asnNum, asnNumHref=asnNumHref, asnName=asnName, ipv4NumIps=ipv4NumIps,
                              ipv6NumIps=ipv6NumIps, countryCode=response.meta["countryCode"])

            # print(asnItem)
            yield asnItem
            # yield Request(asnNumHref, dont_filter=False,
            #               callback=lambda response, asnNum=asnNum: self.asn_content_parse(response, asnNum))
            yield Request(asnNumHref, meta={"asnNum": asnNum}, callback=self.asn_content_parse, dont_filter=False)

    def asn_content_parse(self, response):

        try:
            asn_content_list = response.xpath('//div[@class="col-sm-3"]')

            if len(asn_content_list.xpath('./h5/text()').extract()) == 2:
                asName = "".join(asn_content_list.xpath('./h5/text()').extract()[0])
                orgName = "".join(asn_content_list.xpath('./h5/text()').extract()[1])
            else:
                asName = ""
                orgName = "".join(asn_content_list.xpath('./h5/text()').extract()[0])
            registryRegion = "".join(asn_content_list.xpath('.//a/text()').extract())
            ipv4Prefixes = "".join(asn_content_list.xpath('./p/text()').extract()[0])
            ipv6Prefixes = "".join(asn_content_list.xpath('./p/text()').extract()[1])
            ipv4NumIps = "".join(asn_content_list.xpath('./p/text()').extract()[2])
            ipv6NumIps = "".join(asn_content_list.xpath('./p/text()').extract()[3])

            cidr_tr_list = response.xpath('//*[@id="pills-ipv4"]//table/tr')
            for cidr_tr in cidr_tr_list:
                td = cidr_tr.xpath('td')
                cidr = "".join(td[0].xpath('a/text()').extract())
                cidrHref = "".join(td[0].xpath('a/@href').extract())
                cidrHref = response.urljoin(cidrHref)
                cidrDescription = "".join(td[1].xpath('./text()').extract())
                ipNum = "".join(td[2].xpath('./text()').extract())

                asnContentItem = ASNContentItem(asnNum=response.meta["asnNum"], asName=asName, orgName=orgName,
                                                registryRegion=registryRegion, ipv4Prefixes=ipv4Prefixes,
                                                ipv6Prefixes=ipv6Prefixes,
                                                ipv4NumIps=ipv4NumIps, ipv6NumIps=ipv6NumIps, cidr=cidr,
                                                cidrHref=cidrHref, cidrDescription=cidrDescription, ipNum=ipNum)
                yield asnContentItem
        except Exception as e:
            print("信息不全呗，不处理你了，滚蛋")
